import os
import re

import requests
from lxml import etree
import openpyxl

areas = ['禅城', '南海', '顺德', '高明', '三水']
building_workbook = openpyxl.load_workbook("新盘交易量表格.xlsx")
building_sheet = building_workbook["Sheet1"]

pmip_str = '' # 代理ip地址及端口 如 192.168.0.2:8000
if pmip_str:
    pmip = proxy = {'http': ('http://' + pmip_str),
                    'https': ('https://' + pmip_str)}

try:
    os.mkdir("./htmlCode")
except:
    print("已存在htmlCode文件夹")

def download_url( page):
    """
    保存网页信息到htmlCode文件夹中，以页码命名
    :param url:网址链接
    :param page:页码
    :return:
    """
    if pmip_str:
        if page == 1:
            r = requests.get(url="https://fsfc.fszj.foshan.gov.cn/search/index.do?sw=&dn=&lx=&hx=&mj=&jg=&od=-SORT%3B-STIME%3B-LASTMODIFYTIME%3BRELEVANCE&ys=", proxies=pmip)
        else:
            r = requests.get(url="https://fsfc.fszj.foshan.gov.cn/search/index.do?sw=&dn=&hx=&lx=&mj=&jg=&ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE&p=" + str(page), proxies=pmip)
    else:
        if page == 1:
            r = requests.get(url="https://fsfc.fszj.foshan.gov.cn/search/index.do?sw=&dn=&lx=&hx=&mj=&jg=&od=-SORT%3B-STIME%3B-LASTMODIFYTIME%3BRELEVANCE&ys=")
        else:
            r = requests.get(url="https://fsfc.fszj.foshan.gov.cn/search/index.do?sw=&dn=&hx=&lx=&mj=&jg=&ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE&p=" + str(page))
    f = open("./htmlCode/" + str(page) + '.html', 'w', encoding='utf-8')
    # f.write(r.text)
    f.write(r.text.replace('<em>', '').replace('</em>', ''))
    f.close()
    print("第" + str(page) + "页保存成功")

def read_html(page):
    html = etree.parse("./htmlCode/" + str(page) + '.html', etree.HTMLParser())
    buildingNames = html.xpath('//div[@class="new-house"]//h3/a/text()') # 楼盘名
    print(buildingNames)
    buildingAddresses = html.xpath('//div[@class="new-house"]//p[1]/text()') # 楼盘地址
    print(buildingAddresses)
    developers = html.xpath('//div[@class="new-house"]//p[2]/text()') # 开发商
    print(developers)
    propertyCompanys = html.xpath('//div[@class="new-house"]//p[3]/text()') # 物业公司
    print(propertyCompanys)
    soldInfos = html.xpath('//div[@class="new-house"]//p[4]/text()')  # 售楼信息
    print(soldInfos)
    # soldOutNums = html.xpath('//div[@class="new-house"]//h5[1]/text()') # 已售套数
    # print(soldOutNums)
    # forSaleNums = html.xpath('//div[@class="new-house"]//h5[2]/text()') # 未售套数
    # print(forSaleNums)
    if len(buildingNames) == len(buildingAddresses) == len(developers) == len(propertyCompanys) == len(soldInfos):
        print("第" + str(page) + "页数据正确")
        return (buildingNames, buildingAddresses, developers, propertyCompanys, soldInfos)
    else:
        print("第" + str(page) + "页数据正确")
        raise str("数据错误，请检查")

def writeExcel(page):
    buildingNames, buildingAddresses, developers, propertyCompanys, soldInfos = read_html(page)
    i = 0
    for buildingName in buildingNames:
        buildingName = buildingName.strip() # 楼盘名
        buildingAddress = buildingAddresses[i].strip("楼盘地址： ").strip(r"\t\t   \t").strip() # 楼盘地址
        developer = developers[i].strip("开  发  商：").strip() # 开发商
        propertyCompany = propertyCompanys[i].strip("物业公司：").strip()  # 物业公司
        soldInfo = soldInfos[i]  # 售楼信息
        p1 = re.compile(r'[【](.*?)[】]', re.S)  # 正则最小匹配
        soldOutNum,  forSaleNum = re.findall(p1, soldInfo) # 已售套数 未售套数

        # 已售套数 未售套数 从字符类型转换成整型
        try:
            soldOutNum = int(soldOutNum)
        except:
            pass
        try:
            forSaleNum = int(forSaleNum)
        except:
            pass

        areaName = '' # 区域名称
        for area in areas: # 获取区域
            if area in buildingAddress:
                areaName = "佛山市" + area + "区"
                # buildingAddress = buildingAddress.strip(area)
        xlsx_row = building_sheet.max_row + 1 # 在excel第几行写入
        building_sheet.cell(row=xlsx_row, column=1).value = buildingName
        building_sheet.cell(row=xlsx_row, column=2).value = areaName
        building_sheet.cell(row=xlsx_row, column=3).value = buildingAddress
        building_sheet.cell(row=xlsx_row, column=4).value = soldOutNum
        building_sheet.cell(row=xlsx_row, column=5).value = forSaleNum
        building_sheet.cell(row=xlsx_row, column=6).value = page

        building_workbook.save('新盘交易量表格.xlsx')
        i = i + 1



if __name__ == '__main__':
    # page = 8
    # download_url(page)
    # read_html(page)
    # writeExcel(page)

    for page in range(5, 6): # 要爬取的页码区间，包头不包尾
        download_url(page) # 下载并保存成html文件
        writeExcel(page) # 写入Excel数据
        pass
